
import re


from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.keys import Keys
import time


def printSource(browser):
    print browser.page_source

def getBrowser(url):
    
    browser = webdriver.Firefox() # Get local session of firefox
    browser.get(url) # Load page
##        print browser.page_source
    return browser

def getUrlInfo(url, extractor=printSource):
    try:
        browser = webdriver.Firefox() # Get local session of firefox
        browser.get(url) # Load page
        extractor(browser)
##        print browser.page_source
    finally:
        browser.close()

def getViewPageSource(url):
    global page_source
    page_source = None
    def f(b):
        global page_source
        page_source = b.page_source
        
    getUrlInfo(url, f)
    return page_source

pages = {}

def getViewPageSourceWithCache(url):
    ret = pages.get(url)
    if (ret == None):
        ret = getViewPageSource(url)
        pages[url] = ret
    return ret


def getHtml(url):
    import lxn
    __STD_DB_PATH__ = "../lxdb/tag.db"
    __STD_RS_PATH__ = "../lxdb/"
    global hc
    if hc == None:
        hc = lxn.UrlContentCache(__STD_DB_PATH__, __STD_RS_PATH__)
    return hc.getHtml(url, lxn._ORACLE_PROXY_)[0]

def getHtml1(url):
    return getViewPageSourceWithCache(url)


def pachong_org1(url="http://pachong.org"):
    global data
    data = None
    def ext(a):
        global data
        bb = a.find_elements_by_xpath("//tr[@data-id]")
        data = map(lambda e:e.text, bb)
    getUrlInfo(url, ext)
    return data

def xici1(url = "http://www.xici.net.co"):
    global data
    data = None
    def ext(a):
        global data
        bb = a.find_elements_by_xpath("//tr")
        data = map(lambda e:e.text, bb)
    getUrlInfo(url, ext)
    import string
    data = map(lambda e:e if e[0] in string.digits else None, data)
    return data



def pachong_org(url = "http://pachong.org"):
    html = getHtml1(url)
    #print html

    #fetch whole list
    pat = """<tr data-id[\w\W]+?</tr>"""
    proxy_list = re.compile(pat).findall(html)

    print proxy_list[0]

    #fetch address
    p_addr = """\d+\.\d+\.\d+\.\d"""
    pp_addr = re.compile(p_addr)
    addresses = map(lambda e:pp_addr.findall(e)[0], proxy_list)
    print addresses

    #fetch port
    p_port = """</script>\s*"\d+"</td>"""
    pp_port = re.compile(p_port)
    ports = map(lambda e:pp_port.findall(e)[0], proxy_list)
    print ports

def xici(url = "www.xici.net.co"):
    html = getHtml1(url)
    p_ip = """<td>(\d+\.\d+\.\d+\.\d+)</td>"""
    p_port = """<td>(\d+)<\td>"""
    p_skip = """<td>[\w\W]+?</td>"""
    p_type = """<td>(H[A-z]+)</td>"""
    p_total = p_ip + p_port + p_skip + p_skip + p_type
    pp_total = re.compile(p_total)
    print html
    contents = pp_total.findall(html)
    print contents[0:8]
    
if __name__ == "__main__":
##    pachong_org()
    #xici()

    #lxn.getHtml("www.xici.net.co", "rn-proxy.oracle.com", 80)
    d = pachong_org1()
    dd = xici1()
    import sys
    if len(sys.argv)!=2: print "Supply only the file name to output"
    with open(sys.argv[1], "wb") as f:
        for e in d:
            if e==None: continue
            f.write(e.encode("utf-8"))
            f.write("\n")
	for e in dd:
            if e==None: continue
	    f.write(e.encode("utf-8"))
            f.write("\n")

